# coding: UTF-8
# Filename : logToDb.py
# Company : imedia
# Author : rose
# -*-coding:utf-8 -*-
'''im_log im_logs记录入库 初次入库时间2011-07-19'''
import mysql, utils, log
import codecs, sys, os, time
import threading

date1 = utils.getNow()
print date1

#检测线程类 用于监控所有线程 反馈最终信息
class CheckThread(threading.Thread):
    def __init__(self, param, n):
        threading.Thread.__init__(self)
        self.param = param
        self.n = n
    def run(self):
        self.doCheck()
    def doCheck(self):
        while True:
            count = self.param.activeCount()
            #count=2时候只有本线程和主线程
            if count <= 2:
                date2 = utils.getNow()
                print date2
                msg = '所有入库线程结束' + '共入库' + str(self.n) + '条,耗时' + str(date2-date1) 
                print msg
                log.info(msg)
                break
#入库线程类
class InsertThread(threading.Thread):
    def __init__(self, sql, params, flag = 'Daemon'):
        self.sql = sql
        self.params = params
        self.flag = flag
        threading.Thread.__init__(self)

    def run(self):
        self.insertlog()
    
    #执行插入逻辑
    def insertlog(self):
        print '线程' + self.flag + '开启'
        mysql.insertMany(self.sql, self.params)
        print '线程' + self.flag + '入库成功'

'----------------以下做分析入库----------------'

del_im_log = '''delete from im_log where  time >= %s and time <= %s;
'''
del_im_logs = '''delete from im_log_s where  time >= %s and time <= %s;
'''

sql_im_log = '''insert  into `im_log`
(`ip`, `user_id`, `time`, `type`, `phone_brand`, `phone_sys`,
 `phone_browser`, `province_id`, `city_id`, `p_id`, `cus_id`, `media_id`) 
values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
'''

sql_im_logs = '''insert  into `im_log_s`
(`cus_id`, `time`, `p_id`, `media_id`) 
values (%s, %s, %s, %s)
'''
#日期设定
yesterday = str(utils.getYesterday())
if len(sys.argv) >= 2:
    yesterday = sys.argv[1]

charset = utils.readFromIni('charset')

start_time_str = yesterday + ' 00:00:00'
end_time_str = yesterday + ' 23:59:59'
format = '%Y-%m-%d %H:%M:%S'
start_time = utils.strToTimestamp(start_time_str, format)
end_time = utils.strToTimestamp(end_time_str, format)


n = 10000    
def insert_logic(fileName, sql, table = ''):
    file_Name = fileName+"."+yesterday
    if not os.path.exists(file_Name):
        print '没有发现 ' + file_Name
        exit(0)
    #删除入库数据
    #mysql.delete(del_im_log, [start_time, end_time])
    #mysql.delete(del_im_logs, [start_time, end_time])
    rstlist =codecs.open(file_Name,'r',charset)
    i = 0
    params = []
    for line in rstlist:
        line = str(line).replace('\n', '')
        arrays = line.split('`')
        arrays = arrays[1:]
        #做空字符串处理
        temp = []
        for j in range(len(arrays)):
            obj_temp = str(arrays[j])
            if obj_temp.strip() == '': obj_temp = None
            temp.append(obj_temp)
        #转化为元组
        arrays = tuple(temp)
        params.append(arrays)
        i += 1
        #n条数据批量入库
        if i % n == 0:
            insertThread = InsertThread(sql, params, table + ':' + str(int(i / n)))
            insertThread.start()
            if int(i / n) % 10 == 0:
                print '休息10秒'
                time.sleep(10)
                print '继续'
            params = []
    insertThread = InsertThread(sql, params, table + ':last')
    insertThread.start()
    return i
    
im_log_path = utils.readFromIni('im_log_path')
im_logs_path = utils.readFromIni('im_logs_path')
#im_log入库
a = insert_logic(im_log_path, sql_im_log, 'im_log')
#im_logs入库
b = insert_logic(im_logs_path, sql_im_logs, 'im_log_s')

checkThread = CheckThread(threading, a+b)
checkThread.start()
